{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 0. 环境设定"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\Program Files\\Anaconda\\lib\\site-packages\\h5py\\__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n"
     ]
    }
   ],
   "source": [
    "import argparse\n",
    "import sys\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "from tensorflow.examples.tutorials.mnist import input_data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1. 数据准备"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting ../data/mnist/input_data/train-images-idx3-ubyte.gz\n",
      "Extracting ../data/mnist/input_data/train-labels-idx1-ubyte.gz\n",
      "Extracting ../data/mnist/input_data/t10k-images-idx3-ubyte.gz\n",
      "Extracting ../data/mnist/input_data/t10k-labels-idx1-ubyte.gz\n"
     ]
    }
   ],
   "source": [
    "data_dir = '../data/mnist/input_data/'\n",
    "mnist = input_data.read_data_sets(data_dir, one_hot=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((55000, 784), (55000, 10))"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mnist.train.images.shape,mnist.train.labels.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((10000, 784), (10000, 10))"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mnist.test.images.shape,mnist.test.labels.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2. 准备placeholder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "X=tf.placeholder(tf.float32,[None,784],name='X_placeholder')\n",
    "Y=tf.placeholder(tf.float32,[None,10],name='Y_placeholder')\n",
    "init_learning_rate = tf.placeholder(tf.float32)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3. 准备参数/权重"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_hidden1=100\n",
    "n_hidden2=10\n",
    "n_input=784\n",
    "n_classes=10\n",
    "\n",
    "weights={\n",
    "    'W1':tf.Variable(tf.random_normal([n_input,n_hidden1]),name='W1'),\n",
    "    'W2':tf.Variable(tf.random_normal([n_hidden1,n_hidden2]),name='W2'),\n",
    "}\n",
    "biases={\n",
    "    'b1':tf.Variable(tf.random_normal([n_hidden1]),name='b1'),\n",
    "    'b2':tf.Variable(tf.random_normal([n_hidden2]),name='b2'),\n",
    "}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 4. 构建计算graph"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def mutilayer_perception(x,weights,biases):\n",
    "    logits_1=tf.add(tf.matmul(x,weights['W1']),biases['b1'])\n",
    "    output_1=tf.nn.sigmoid(logits_1)\n",
    "    logits_2=tf.add(tf.matmul(output_1,weights['W2']),biases['b2'])\n",
    "    out_layer=logits_2\n",
    "    return out_layer\n",
    "\n",
    "pred=mutilayer_perception(X,weights,biases)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 5. 计算loss和accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-8-8ba921707586>:1: softmax_cross_entropy_with_logits (from tensorflow.python.ops.nn_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "\n",
      "Future major versions of TensorFlow will allow gradients to flow\n",
      "into the labels input on backprop by default.\n",
      "\n",
      "See tf.nn.softmax_cross_entropy_with_logits_v2.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "cross_entropy=tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=Y,logits=pred))\n",
    "l2_loss=tf.nn.l2_loss(weights['W1'])+tf.nn.l2_loss(weights['W2'])\n",
    "total_loss=cross_entropy+4e-5*l2_loss\n",
    "\n",
    "correct_prediction = tf.equal(tf.argmax(pred, 1), tf.argmax(Y, 1))\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 6. 设置optimizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "#设置learning_rate迭代方式\n",
    "epoch_steps = tf.to_int64(tf.div(60000, tf.shape(X)[0]))\n",
    "global_step = tf.train.get_or_create_global_step()\n",
    "current_epoch = global_step//epoch_steps\n",
    "decay_times = current_epoch \n",
    "current_learning_rate = tf.multiply(init_learning_rate, \n",
    "                                    tf.pow(0.65, tf.to_float(decay_times)))\n",
    "#设置optimizer\n",
    "optimizer = tf.train.AdamOptimizer(current_learning_rate)\n",
    "gradients = optimizer.compute_gradients(total_loss)\n",
    "train_step = optimizer.apply_gradients(gradients)\n",
    "\n",
    "train_step = tf.train.AdamOptimizer(\n",
    "    current_learning_rate).minimize(\n",
    "    total_loss, global_step=global_step)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 7. 初始化变量"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "init=tf.global_variables_initializer()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 8. 在session中执行graph"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "step 100, entropy loss: 0.681594, l2_loss: 27472.996094, total loss: 1.780514, current_lr_value:0.010000\n",
      "0.8345\n",
      "step 200, entropy loss: 0.367884, l2_loss: 23064.574219, total loss: 1.290467, current_lr_value:0.010000\n",
      "0.8857\n",
      "step 300, entropy loss: 0.595597, l2_loss: 20459.265625, total loss: 1.413967, current_lr_value:0.010000\n",
      "0.9031\n",
      "step 400, entropy loss: 0.478826, l2_loss: 18501.800781, total loss: 1.218898, current_lr_value:0.010000\n",
      "0.9227\n",
      "step 500, entropy loss: 0.335434, l2_loss: 16830.900391, total loss: 1.008670, current_lr_value:0.010000\n",
      "0.925\n",
      "step 600, entropy loss: 0.062877, l2_loss: 15374.311523, total loss: 0.677850, current_lr_value:0.010000\n",
      "0.935\n",
      "step 700, entropy loss: 0.273189, l2_loss: 14483.599609, total loss: 0.852533, current_lr_value:0.006500\n",
      "0.9343\n",
      "step 800, entropy loss: 0.122981, l2_loss: 13645.873047, total loss: 0.668816, current_lr_value:0.006500\n",
      "0.9427\n",
      "step 900, entropy loss: 0.143765, l2_loss: 12854.164062, total loss: 0.657932, current_lr_value:0.006500\n",
      "0.9429\n",
      "step 1000, entropy loss: 0.182816, l2_loss: 12113.699219, total loss: 0.667364, current_lr_value:0.006500\n",
      "0.9466\n",
      "step 1100, entropy loss: 0.168540, l2_loss: 11422.833008, total loss: 0.625453, current_lr_value:0.006500\n",
      "0.9471\n",
      "step 1200, entropy loss: 0.090568, l2_loss: 10772.275391, total loss: 0.521459, current_lr_value:0.006500\n",
      "0.9504\n",
      "step 1300, entropy loss: 0.248016, l2_loss: 10355.576172, total loss: 0.662239, current_lr_value:0.004225\n",
      "0.9529\n",
      "step 1400, entropy loss: 0.134838, l2_loss: 9945.517578, total loss: 0.532658, current_lr_value:0.004225\n",
      "0.9525\n",
      "step 1500, entropy loss: 0.133817, l2_loss: 9552.774414, total loss: 0.515928, current_lr_value:0.004225\n",
      "0.9528\n",
      "step 1600, entropy loss: 0.123909, l2_loss: 9170.945312, total loss: 0.490747, current_lr_value:0.004225\n",
      "0.9552\n",
      "step 1700, entropy loss: 0.165877, l2_loss: 8812.005859, total loss: 0.518357, current_lr_value:0.004225\n",
      "0.9566\n",
      "step 1800, entropy loss: 0.170683, l2_loss: 8468.490234, total loss: 0.509423, current_lr_value:0.004225\n",
      "0.9566\n",
      "step 1900, entropy loss: 0.060558, l2_loss: 8241.584961, total loss: 0.390222, current_lr_value:0.002746\n",
      "0.9582\n",
      "step 2000, entropy loss: 0.060049, l2_loss: 8015.598633, total loss: 0.380673, current_lr_value:0.002746\n",
      "0.9596\n",
      "step 2100, entropy loss: 0.051689, l2_loss: 7797.172852, total loss: 0.363576, current_lr_value:0.002746\n",
      "0.9603\n",
      "step 2200, entropy loss: 0.083615, l2_loss: 7582.230469, total loss: 0.386904, current_lr_value:0.002746\n",
      "0.9602\n",
      "step 2300, entropy loss: 0.093771, l2_loss: 7371.910156, total loss: 0.388647, current_lr_value:0.002746\n",
      "0.9604\n",
      "step 2400, entropy loss: 0.128666, l2_loss: 7167.997070, total loss: 0.415386, current_lr_value:0.002746\n",
      "0.9622\n",
      "step 2500, entropy loss: 0.053642, l2_loss: 7034.142578, total loss: 0.335008, current_lr_value:0.001785\n",
      "0.9604\n",
      "step 2600, entropy loss: 0.081586, l2_loss: 6902.298828, total loss: 0.357678, current_lr_value:0.001785\n",
      "0.9622\n",
      "step 2700, entropy loss: 0.039516, l2_loss: 6770.560547, total loss: 0.310339, current_lr_value:0.001785\n",
      "0.9637\n",
      "step 2800, entropy loss: 0.056456, l2_loss: 6641.643066, total loss: 0.322122, current_lr_value:0.001785\n",
      "0.9621\n",
      "step 2900, entropy loss: 0.077601, l2_loss: 6514.568359, total loss: 0.338184, current_lr_value:0.001785\n",
      "0.9631\n",
      "step 3000, entropy loss: 0.109238, l2_loss: 6387.166016, total loss: 0.364725, current_lr_value:0.001785\n",
      "0.9612\n",
      "step 3100, entropy loss: 0.075595, l2_loss: 6303.846191, total loss: 0.327749, current_lr_value:0.001160\n",
      "0.9631\n",
      "step 3200, entropy loss: 0.059080, l2_loss: 6219.891602, total loss: 0.307875, current_lr_value:0.001160\n",
      "0.9642\n",
      "step 3300, entropy loss: 0.115356, l2_loss: 6137.932129, total loss: 0.360873, current_lr_value:0.001160\n",
      "0.9637\n",
      "step 3400, entropy loss: 0.045363, l2_loss: 6056.969238, total loss: 0.287642, current_lr_value:0.001160\n",
      "0.9646\n",
      "step 3500, entropy loss: 0.058064, l2_loss: 5976.362305, total loss: 0.297118, current_lr_value:0.001160\n",
      "0.9644\n",
      "step 3600, entropy loss: 0.025187, l2_loss: 5896.164062, total loss: 0.261033, current_lr_value:0.001160\n",
      "0.9639\n",
      "step 3700, entropy loss: 0.087604, l2_loss: 5843.687012, total loss: 0.321352, current_lr_value:0.000754\n",
      "0.9649\n",
      "step 3800, entropy loss: 0.071927, l2_loss: 5790.982910, total loss: 0.303567, current_lr_value:0.000754\n",
      "0.9651\n",
      "step 3900, entropy loss: 0.056096, l2_loss: 5738.657227, total loss: 0.285642, current_lr_value:0.000754\n",
      "0.9647\n",
      "step 4000, entropy loss: 0.055185, l2_loss: 5685.082520, total loss: 0.282588, current_lr_value:0.000754\n",
      "0.9644\n",
      "step 4100, entropy loss: 0.048314, l2_loss: 5633.197266, total loss: 0.273642, current_lr_value:0.000754\n",
      "0.9639\n",
      "step 4200, entropy loss: 0.059923, l2_loss: 5580.631836, total loss: 0.283148, current_lr_value:0.000754\n",
      "0.9653\n",
      "step 4300, entropy loss: 0.071014, l2_loss: 5546.596191, total loss: 0.292878, current_lr_value:0.000490\n",
      "0.965\n",
      "step 4400, entropy loss: 0.033072, l2_loss: 5512.405273, total loss: 0.253568, current_lr_value:0.000490\n",
      "0.9654\n",
      "step 4500, entropy loss: 0.050355, l2_loss: 5478.024902, total loss: 0.269476, current_lr_value:0.000490\n",
      "0.965\n",
      "step 4600, entropy loss: 0.045887, l2_loss: 5444.013672, total loss: 0.263647, current_lr_value:0.000490\n",
      "0.9657\n",
      "step 4700, entropy loss: 0.044398, l2_loss: 5409.240723, total loss: 0.260768, current_lr_value:0.000490\n",
      "0.9659\n",
      "step 4800, entropy loss: 0.037075, l2_loss: 5375.308594, total loss: 0.252088, current_lr_value:0.000490\n",
      "0.9651\n",
      "step 4900, entropy loss: 0.080333, l2_loss: 5352.610840, total loss: 0.294437, current_lr_value:0.000319\n",
      "0.9665\n",
      "step 5000, entropy loss: 0.047238, l2_loss: 5330.312988, total loss: 0.260450, current_lr_value:0.000319\n",
      "0.9655\n"
     ]
    }
   ],
   "source": [
    "with tf.Session() as sess:\n",
    "    sess.run(init)\n",
    "    for step in range(5000):\n",
    "        batch_x,batch_y= mnist.train.next_batch(100)\n",
    "        lr = 1e-2\n",
    "        _, loss, l2_loss_value, total_loss_value, current_lr_value = \\\n",
    "        sess.run([train_step, cross_entropy, l2_loss, total_loss, current_learning_rate],\n",
    "               feed_dict={X: batch_x, Y: batch_y, init_learning_rate:lr})\n",
    "  \n",
    "        if (step+1) % 100 == 0:\n",
    "            print('step %d, entropy loss: %f, l2_loss: %f, total loss: %f, current_lr_value:%f' % \n",
    "            (step+1, loss, l2_loss_value, total_loss_value,current_lr_value))\n",
    "    #print(sess.run(accuracy, feed_dict={x: batch_xs, y_: batch_ys}))\n",
    "            print(sess.run(accuracy, feed_dict={X: mnist.test.images, Y: mnist.test.labels}))\n",
    "    #print(current_lr_value)\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
